# Define here the models for your spider middleware
#
# See documentation in:
# https://docs.scrapy.org/en/latest/topics/spider-middleware.html
import os

from scrapy import signals
import json
import time
from selenium import webdriver
from selenium.webdriver.common.desired_capabilities import DesiredCapabilities
from scrapy.utils.conf import closest_scrapy_cfg
import os
proj_root = closest_scrapy_cfg()
assets_path=os.path.abspath(os.path.dirname('proj_root'))+'\\assets'
# useful for handling different item types with a single interface
from itemadapter import is_item, ItemAdapter

capabilities = DesiredCapabilities.CHROME
capabilities['goog:loggingPrefs'] = {'browser': 'ALL'}
options = webdriver.ChromeOptions()
options.add_argument('--disable-blink-features=AutomationControlled')
# options.add_argument('--headless')
# options.add_experimental_option('excludeSwitches', ['enable-automation'])
# options.add_argument('--disable-gpu')
driver = webdriver.Chrome(desired_capabilities=capabilities, chrome_options=options)

class requestHeaderDecorateMiddleware(object):
    msToken=''
    tt_scid=''
    Bogus=''
    signature=''
    webid=''
    def __init__(self):
        self.driver=driver
        self.file = open('G:/python/scrapy爬取抖音短视频/douyin/assets/douyin1_withkeyword.json', 'r',encoding = 'utf-8')
        self.userId = json.load(self.file)['userId']
        self.sessionid_ss = self.userId['sessionid_ss']
        self.sid_tt = self.userId['sid_tt']
        self.sid_guard = self.userId['sid_guard']

    def process_request(self, request, spider):
        request.headers['Referer'] = 'https://www.douyin.com/'
        if 'https://www.douyin.com/aweme/v1/web/tab/feed/' in request.url:
            refresh_index=request.headers['refresh_index'].decode()
            # 我们需要得到请求的参数,然后根据请求的参数,我们去获取DFS和_
            # 避免被检测
            # driver.execute_cdp_cmd("Page.addScriptToEvaluateOnNewDocument", {
            #     "source": """
            #     Object.defineProperty(navigator, 'webdriver', {
            #       get: () => undefined
            #     })
            #   """
            # })
            if request.headers.get('isFirst').decode()=='True':
                self.driver.get('https://www.douyin.com/')
            time.sleep(5)
            if self.msToken == '':
                self.msToken = self.driver.execute_script('return localStorage.getItem("msToken");')
                self.driver.execute_script('localStorage.setItem("msTokenToken","{}")'.format(self.msToken))
            request.headers['cookie'] = 'msToken={};'.format(self.msToken)
            # 'sid_tt': '6f1cc05bb191a5adf8f42b7fdaefd5bb',
            # sid_tt:sid_guard的原本样式
            # sid_guard:标识用户id
            # sessionid_ss:这三个都是用户的表示id,id会刷新视频
            request.cookies={
                'sessionid_ss': self.sessionid_ss,
                'sid_tt': self.sid_tt,
                'sid_guard': self.sid_guard,
                'msToken': self.msToken,
                }
            # 赋值refresh_index
            self.driver.execute_script('window.isKeyWord=false;window.userId=false;')
            self.driver.execute_script('window.refresh_index=%d;'%int(refresh_index))
            js = open('G:/python/scrapy爬取抖音短视频/douyin/assets/test2.js', 'r').read()  # js语句
            # 执行js的方法
            self.driver.execute_script(js)
            time.sleep(3)
            # 现在判断这个msToken是否存在,存在的话拿我们存在的数据进行赋值,不存在的话
            self.webid=self.driver.execute_script('return window.webid;')  # 执行js的方法
              # 执行js的方法
            self.Bogus=self.driver.execute_script('return window.Bogus;')  # 执行js的方法
            self.signature=self.driver.execute_script('return window._signature;')  # 执行js的方法
            url=str(request.url)
            url=url.replace('webidValue',self.webid).replace('msTokenValue',self.msToken).replace('X-BogusValue',self.Bogus).replace('_signatureValue',self.signature)
            spider.logger.warning('输出获取到的url参数:'+self.Bogus+'|||||||'+self.signature+'||||||||||||||'+self.msToken+'||||||||||||'+self.webid)
            # time.sleep(400)
            spider.logger.warning('输出爬取数据的url:{}'.format(url))
            time.sleep(2)
            # self.driver.quit()
            with open('G:/python/scrapy爬取抖音短视频/douyin/assets/file_list.txt', 'a+') as file:
                file.write(url+'\n')
            request._set_url(url)
            # self.driver.quit()

            # 首先请求
    def process_response(self, request, response, spider):
        if spider.name=='douyin1':
            return response
        return response
    def __del__(self):
        self.driver.quit()

class douyin1WithKeyWordMiddleware(object):
    msToken=''
    tt_scid=''
    Bogus=''
    signature=''
    webid=''
    def __init__(self):
        self.driver=driver
        self.file=open('G:/python/scrapy爬取抖音短视频/douyin/assets/douyin1_withkeyword.json',encoding = 'utf-8')
        self.userId=json.load(self.file)['userId']
        self.sessionid_ss=self.userId['sessionid_ss']
        self.sid_tt=self.userId['sid_tt']
        self.sid_guard=self.userId['sid_guard']

    def process_request(self, request, spider):
        request.headers['Referer'] = 'https://www.douyin.com/'
        if 'https://www.douyin.com/aweme/v1/web/general/search/single/' in request.url:
            # 我们需要得到请求的参数,然后根据请求的参数,我们去获取DFS和_
            if request.headers.get('isFirst').decode()=='True':
                self.driver.get('https://www.douyin.com/')
            time.sleep(5)
            if self.msToken == '':
                self.msToken = self.driver.execute_script('return localStorage.getItem("msToken");')
                self.driver.execute_script('localStorage.setItem("msTokenToken","{}")'.format(self.msToken))
            request.headers['cookie'] = 'msToken={};'.format(self.msToken)
            # 'sid_tt': '6f1cc05bb191a5adf8f42b7fdaefd5bb',
            # sid_tt:sid_guard的原本样式
            # sid_guard:标识用户id
            # sessionid_ss:这三个都是用户的表示id,id会刷新视频
            request.cookies={
                'sessionid_ss': self.sessionid_ss,
                'sid_tt': self.sid_tt,
                'sid_guard': self.sid_guard,
                'msToken': self.msToken,
                }
            # 赋值refresh_index
            self.driver.execute_script('window.isKeyWord=true;')
            # 赋值参数
            self.driver.execute_script('window.keyword="{}";window.offset="{}";window.count="{}";window.is_filter_search="{}";window.query_correct_type="{}";window.sort_type="{}";'.format(request.headers.get('keyword').decode(),request.headers.get('offset').decode(),request.headers.get('count').decode(),request.headers.get('is_filter_search').decode(),request.headers.get('query_correct_type').decode(),request.headers.get('sort_type').decode()))
            js = open('G:/python/scrapy爬取抖音短视频/douyin/assets/test2.js', 'r').read()  # js语句
            # 执行js的方法
            self.driver.execute_script(js)
            time.sleep(3)
            # 现在判断这个msToken是否存在,存在的话拿我们存在的数据进行赋值,不存在的话
            self.webid=self.driver.execute_script('return window.webid;')  # 执行js的方法
              # 执行js的方法
            self.Bogus=self.driver.execute_script('return window.Bogus;')  # 执行js的方法
            self.signature=self.driver.execute_script('return window._signature;')  # 执行js的方法
            url=str(request.url)
            url=url.replace('webidValue',self.webid).replace('msTokenValue',self.msToken).replace('X-BogusValue',self.Bogus).replace('_signatureValue',self.signature)
            spider.logger.warning("输出获取到的url参数:"+self.Bogus+'|||||||'+self.signature+'||||||||||||||'+self.msToken+'||||||||||||'+self.webid)
            # time.sleep(400)
            print(url)
            spider.logger.warning('输出爬取数据的url:{}'.format(url))
            time.sleep(2)
            # self.driver.quit()
            with open('G:/python/scrapy爬取抖音短视频/douyin/assets/file_list.txt', 'a+') as file:
                file.write(url+'\n')
            request._set_url(url)
            # self.driver.quit()

            # 首先请求
    def process_response(self, request, response, spider):
        if spider.name=='douyin1_withkeyword':
            return response
        return response

    def __del__(self):
        self.file.close()
        self.driver.quit()

class douyin1UserIdMiddleware(object):
    msToken=''
    tt_scid=''
    Bogus=''
    signature=''
    webid=''
    def __init__(self):
        self.driver=driver
        self.file = open('G:/python/scrapy爬取抖音短视频/douyin/assets/douyin1_withkeyword.json', 'r',encoding = 'utf-8')
        self.userId = json.load(self.file)['userId']
        self.sessionid_ss = self.userId['sessionid_ss']
        self.sid_tt = self.userId['sid_tt']
        self.sid_guard = self.userId['sid_guard']

    def process_request(self, request, spider):
        request.headers['Referer'] = 'https://www.douyin.com/'
        if 'https://www.douyin.com/aweme/v1/web/aweme/post/' in request.url:
            count=request.headers['count'].decode()
            sec_user_id=request.headers['sec_user_id'].decode()
            max_cursor=request.headers['max_cursor'].decode()
            isFirst=request.headers['isFirst'].decode()
            if isFirst=='True':
                self.driver.get('https://www.douyin.com/')
            time.sleep(5)
            if self.msToken == '':
                self.msToken = self.driver.execute_script('return localStorage.getItem("msToken");')
                self.driver.execute_script('localStorage.setItem("msTokenToken","{}")'.format(self.msToken))
            request.headers['cookie'] = 'msToken={};'.format(self.msToken)
            # 'sid_tt': '6f1cc05bb191a5adf8f42b7fdaefd5bb',
            # sid_tt:sid_guard的原本样式
            # sid_guard:标识用户id
            # sessionid_ss:这三个都是用户的表示id,id会刷新视频
            request.cookies={
                'sessionid_ss': self.sessionid_ss,
                'sid_tt': self.sid_tt,
                'sid_guard': self.sid_guard,
                'msToken': self.msToken,
                }
            self.driver.execute_script('window.isKeyWord=false;window.userId=true')
            # 赋值refresh_index
            self.driver.execute_script('window.count="{}";'.format(count))
            self.driver.execute_script('window.sec_user_id="{}";'.format(sec_user_id))
            self.driver.execute_script('window.max_cursor="{}";'.format(max_cursor))
            js = open('G:/python/scrapy爬取抖音短视频/douyin/assets/test2.js', 'r').read()  # js语句
            # 执行js的方法
            self.driver.execute_script(js)
            time.sleep(3)
            # 现在判断这个msToken是否存在,存在的话拿我们存在的数据进行赋值,不存在的话
            self.webid=self.driver.execute_script('return window.webid;')  # 执行js的方法
              # 执行js的方法
            self.Bogus=self.driver.execute_script('return window.Bogus;')  # 执行js的方法
            self.signature=self.driver.execute_script('return window._signature;')  # 执行js的方法
            url=str(request.url)
            url=url.replace('webidValue',self.webid).replace('msTokenValue',self.msToken).replace('X-BogusValue',self.Bogus).replace('_signatureValue',self.signature)
            spider.logger.warning('输出获取到的url参数:'+self.Bogus+'|||||||'+self.signature+'||||||||||||||'+self.msToken+'||||||||||||'+self.webid)
            # time.sleep(400)
            spider.logger.warning('输出爬取数据的url:{}'.format(url))
            time.sleep(2)
            with open('G:/python/scrapy爬取抖音短视频/douyin/assets/file_list.txt', 'a+') as file:
                file.write(url+'\n')
            request._set_url(url)
            # self.driver.quit()

            # 首先请求
    def process_response(self, request, response, spider):
        if spider.name=='douyin1_userId':
            return response
        return response
    def __del__(self):
        self.driver.quit()

